/**
 * 
 */
package com.dhcc.cdbp.util;

import java.io.IOException;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.junit.Assert;


/**
 * @author 谢进军
 *
 */
public class AnalyzerUtils {
	public static void main(String[] args) throws IOException {
		String testStr = "java <B>Hello</B> world";
		Analyzer analyzer = new WhitespaceAnalyzer();
		displayTokens(analyzer,testStr);
	}
	public static void displayTokens(Analyzer analyzer,String text) throws IOException{
		TokenStream tokenStream = analyzer.tokenStream("text", text);
		System.out.println("文本：[" + text + "]");
		displayTokens(tokenStream);
		
	}
	public static void displayTokens(TokenStream tokenStream) throws IOException{
		//Lucene内置的语汇单元属性：
		//CharTermAttribute:语汇单元对应的文本
		//PositionIncrementAttribute:位置增量（默认值为1）
		//OffsetAttribute:起始字符和终止字符的偏移量
		//TypeAttribute:语汇单元类型（默认为word）
		//FlagsAttribute:自定义标识位
		//PayloadAttribute:每个语汇单元的byte[]类型有效负载
		OffsetAttribute offsetAttribute = tokenStream.addAttribute(OffsetAttribute.class);
		PositionIncrementAttribute positionIncrementAttribute = tokenStream.addAttribute(PositionIncrementAttribute.class);
		CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
		TypeAttribute typeAttribute = tokenStream.addAttribute(TypeAttribute.class);
		tokenStream.reset();
		int position = 0;
		while (tokenStream.incrementToken()) {
			int increment = positionIncrementAttribute.getPositionIncrement();
			if (increment > 0) {
				position = position + increment;
				System.out.print(position + ":");
			}
			 int startOffset = offsetAttribute.startOffset();
			    int endOffset = offsetAttribute.endOffset();
			    String term = charTermAttribute.toString();
			    System.out.println("[" + term + "]" + ":(" + startOffset + "-->" + endOffset + "):" + typeAttribute.type());
		}
	}
	/**
	 * 断言分词结果
	 * @param analyzer
	 * @param text        源字符串
	 * @param expecteds   期望分词后结果
	 * @throws IOException 
	 */
	public static void assertAnalyzerTo(Analyzer analyzer,String text,String[] expecteds) throws IOException {
		TokenStream tokenStream = analyzer.tokenStream("text", text);
		CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
		for(String expected : expecteds) {
			Assert.assertTrue(tokenStream.incrementToken());
			Assert.assertEquals(expected, charTermAttribute.toString());
		}
		Assert.assertFalse(tokenStream.incrementToken());
		tokenStream.close();
	}
}
